/*
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package net.javlov.example.grid.landmarks;

import java.awt.Point;
import java.util.ArrayList;
import java.util.List;

import javax.swing.Timer;

import net.javlov.Action;
import net.javlov.Agent;
import net.javlov.DecayingLearningRate;
import net.javlov.FixedLearningRate;
import net.javlov.Option;
import net.javlov.Policy;
import net.javlov.QLearningAgent;
import net.javlov.SarsaAgent;
import net.javlov.Simulator;
import net.javlov.example.ExperimentGUI;
import net.javlov.example.grid.Direction;
import net.javlov.example.grid.GridMain;
import net.javlov.example.grid.GridMove;
import net.javlov.example.grid.GridWorldView;
import net.javlov.example.grid.SimpleGridWorld;
import net.javlov.policy.EGreedyPolicy;
import net.javlov.vf.BasicOptionTrace;
import net.javlov.vf.OptionTraces;
import net.javlov.vf.QMap;
import net.javlov.vf.TracedQTable;

public class LandmarksMain extends GridMain {
	
	List<Point> marks;
	int distance = 32;
	
	public static void main(String[] args) {
		LandmarksMain m = new LandmarksMain();
		m.gui = false;
		m.episodes = 1000;
		m.init();
		m.start();
	}
	
	public void makeWorld() {
		marks = new ArrayList<Point>();
		marks.add( new Point(10,80) );
		marks.add( new Point(40,80) );
		marks.add( new Point(40,60) );
		marks.add( new Point(70,60) );
		marks.add( new Point(70,40) );
		
		//world
		world = new SimpleGridWorld(100,100,false);
		world.setObjects(marks);
		world.setGoal(70,40);
	}
	
	public List<Option<? super double[]>> makeActions() {
		//actions
		List<Action> actionPool = new ArrayList<Action>(8);
		actionPool.add( new GridMove(Direction.north, world) );
		actionPool.add( new GridMove(Direction.northeast, world) );
		actionPool.add( new GridMove(Direction.east, world) );
		actionPool.add( new GridMove(Direction.southeast, world) );
		actionPool.add( new GridMove(Direction.south, world) );
		actionPool.add( new GridMove(Direction.southwest, world) );
		actionPool.add( new GridMove(Direction.west, world) );
		actionPool.add( new GridMove(Direction.northwest, world) );
		
		List<Option<? super double[]>> optionPool = new ArrayList<Option<? super double[]>>(marks.size());
		Option o = new ReachLandmarkOption("L1", 1000, marks.get(0), actionPool);
		o.setID(optionPool.size());
		optionPool.add( o );
		
		o = new ReachLandmarkOption("L2", distance, marks.get(1), actionPool);
		o.setID(optionPool.size());
		optionPool.add( o );
		
		o = new ReachLandmarkOption("L3", distance, marks.get(2), actionPool);
		o.setID(optionPool.size());
		optionPool.add( o );
		
		o = new ReachLandmarkOption("L4", distance, marks.get(3), actionPool);
		o.setID(optionPool.size());
		optionPool.add( o );
		
		o = new ReachLandmarkOption("L5", distance, marks.get(4), actionPool);
		o.setID(optionPool.size());
		optionPool.add( o );
		
		return optionPool;
	}
	
	public Agent makeAgent(List<Option<? super double[]>> optionPool) {				
		//value function
		vt = new QMap<double[]>(optionPool, 150); //for TD(0)
		OptionTraces trs = new OptionTraces(new BasicOptionTrace(gamma, lambda,1,false), 0.001);
		TracedQTable<double[]> tqt = new TracedQTable<double[]>( vt, trs);
		
		//policy
		Policy<double[]> pi = new EGreedyPolicy<double[]>(vt, epsilon, optionPool);
		
		//agent
		SarsaAgent<double[]> agent = new QLearningAgent<double[]>(tqt, gamma); //use vt instead of tqt for no el. traces
		agent.setPolicy(pi);
		//agent.setLearnRate( new DecayingLearningRate(1, optionPool.size(), 0.3) );
		agent.setLearnRate( new FixedLearningRate(0.2) );
		agent.setSMDPMode(true);
		agent.setInterruptOptions(true);
		return agent;
	}
	
	public void start() {
		if ( gui ) {
			GridWorldView wv = new LandmarkView(world,10,distance);
			Timer timer = new Timer(1000/24, wv);
			ExperimentGUI g = new ExperimentGUI("Landmark GridWorld", wv, sim);
			sim.suspend();
			timer.start();
			new Thread(this).start();
		} else
			run();
	}

}
